package com.atguigu.gmall.realtime.app.dws;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.atguigu.gmall.realtime.app.func.DimAsyncFunction;
import com.atguigu.gmall.realtime.beans.TradeSkuOrderBean;
import com.atguigu.gmall.realtime.utils.DateFormatUtil;
import com.atguigu.gmall.realtime.utils.MyClickhouseUtil;
import com.atguigu.gmall.realtime.utils.MyKafkaUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;

import java.math.BigDecimal;
import java.util.concurrent.TimeUnit;

/**
 * @author Felix
 * @date 2023/6/16
 * 交易域：sku粒度下单聚合统计
 *      维度：sku
 *      度量：原始金额、实付金额、优惠券减免金额、活动减免金额
 *      数据来源：dwd下单事实表
 * 需要启动的进程
 *      zk、kafka、maxwell、hdfs、hbase、redis、clickhouse、DwdTradeOrderDetail、DwsTradeSkuOrderWindow
 * 开发流程
 *      环境准备
 *      检查点相关的设置
 *      从kafka主题中读取数据
 *      空消息的过滤以及类型转换    jsonStr->jsonObj
 *      去重
 *          为什么会产生空消息以及重复数据？
 *              因为我们从dwd下单事实表中读取数据，下单事实表是将订单明细表、订单表、订单明细活动表、订单明细优惠券表
 *              关联在一起，并且在关联的时候，使用了左外连接，如果左表数据先到，会产生如下数据
 *              左表  null    +I
 *              左表  null    -D
 *              左表  右表     +I
 *              这样的数据发送到kafka主题中，主题会接收以下三条消息
 *              左表  null
 *              null
 *              左表  右表
 *          去重前，按照唯一键(订单明细id)进行分组
 *          去重方式1: 状态 + 定时器
 *              当数据来到的时候，先不往下游传递，只是放到状态中并注册一个5s后执行的定时器，当定时器被触发的时候，
 *              才会将状态中的数据发送到下游。
 *              如果出现重复，重复数据到来的时候，会和状态中存在的数据进行对比，将关联的时候时间大的数据放到状态中
 *          去重方式2：状态 + 抵消
 *              数据来到后，直接放到状态中并发送到下游
 *              如果有重复数据，将状态中影响到度量的属性进行取反，发送到下游
 *      类型转换    jsonObj ->实体类对象
 *      指定Watermark以及提取事件时间字段
 *      按照维度sku进行分组
 *      开窗
 *      聚合计算
 *      维度关联
 *          基本维度关联
 *              PhoenixUtil ->List<T> queryList(String sql,Class<T>clz)
 *              DimUtil     ->getDimInfoNoCache(String tableName,Tuple2 ... params)
 *          优化1：旁路缓存
 *              思路：先从缓存中查询维度数据，如果缓存中存在查询的维度数据，直接将其作为方法的返回值进行返回(缓存命中);
 *                    如果缓存中不存在要查找的维度，发送请求到Phoenix表中查询维度，并将查询的结果放到缓存中，方便下次
 *                    查询使用。
 *              缓存产品选型
 *                   Redis   性能不差、维护性好  √
 *                   状态     性能好、维护性差
 *              Redis中设置
 *                  key:    dim:维度表名:主键值1_主键值2
 *                  type:   string
 *                  expire: 1day
 *                  注意： 如果维度数据发生了变化，需要将Redis中缓存的数据清除掉
 *                  TableProcessFunction->processElement 传递type到下游->
 *                  DimSinkFunction->invoke 执行完upsert后，判断是否对维度做了更新，如果做了更新，调用删除方法
 *          优化2：异步IO
 *              为什么使用异步IO
 *                  在使用map算子对流中数据进行处理的时候，单个并行度上，采用的是同步处理的方式，
 *                  处理完一个元素之后，再处理其它元素
 *                  如果和外部系统进行交互，同步处理有大量的等待时间，性能较低，影响时效性
 *              Flink提供了异步处理的API
 *                  AsyncDataStream.[un]orderedWait(
 *                      流,
 *                      发送异步请求,
 *                      超时时间,
 *                      时间单位
 *                  )
 *                  class DimAsyncFunction extends RichAsyncFunction{
 *                      invoke{
 *                          //开启线程
 *                          //根据流中对象获取要关联的维度的主键(abstract)
 *                          //根据主键获取维度对象
 *                          //将维度对象属性补充到流中对象上(abstract)
 *                          //将补充完维度属性的对象向下游传递
 *                      }
 *                  }
 *                  模板方法设计模式：在父类中定义完成某一个功能的核心算法的骨架，具体的某些步骤延迟到子类中去完成
 *                  在不改变父类核心算法骨架的前提下，每个子类都可以有自己不同的实现
 */
public class DwsTradeSkuOrderWindow {
    public static void main(String[] args) throws Exception {
        //TODO 1.基本环境准备
        //1.1 指定流处理环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //1.2 设置并行度
        env.setParallelism(4);
        //TODO 2.检查点相关设置(略)

        //TODO 3.从kafka主题中读取数据
        //3.1 声明消费的主题以及消费者组
        String topic = "dwd_trade_order_detail";
        String groupId = "dws_trade_sku_order_group";
        //3.2 创建消费者对象
        KafkaSource<String> kafkaSource = MyKafkaUtil.getKafkaSource(topic, groupId);
        //3.3 消费数据 封装为流
        DataStreamSource<String> kafkaStrDS
            = env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "kafka_source");

        //TODO 4.过滤空消息，并对流中的数据进行类型转换     jsonStr->jsonObj
        SingleOutputStreamOperator<JSONObject> jsonObjDS = kafkaStrDS.process(
            new ProcessFunction<String, JSONObject>() {
                @Override
                public void processElement(String jsonStr, Context ctx, Collector<JSONObject> out) throws Exception {
                    if (StringUtils.isNotEmpty(jsonStr)) {
                        out.collect(JSON.parseObject(jsonStr));
                    }
                }
            }
        );
        // {"create_time":"2023-06-05 08:51:37","sku_num":"1","activity_rule_id":"5","split_original_amount":"11999.0000",
        // "split_coupon_amount":"0.0","sku_id":"19","date_id":"2023-06-05","user_id":"2326","province_id":"34",
        // "activity_id":"4","sku_name":"TCL 8","id":"14256542","order_id":"53950","split_activity_amount":"1199.9",
        // "split_total_amount":"10799.1","ts":"1686963097"}
        // jsonObjDS.print(">>>");

        //TODO 5.去重
        //5.1 按照唯一键(订单明细id)进行分组
        KeyedStream<JSONObject, String> orderDetailIdKeyedDS
            = jsonObjDS.keyBy(jsonObj -> jsonObj.getString("id"));
        //5.2 状态 + 定时器
        /*SingleOutputStreamOperator<JSONObject> distinctDS = orderDetailIdKeyedDS.process(
            new KeyedProcessFunction<String, JSONObject, JSONObject>() {
                private ValueState<JSONObject> lastValueState;

                @Override
                public void open(Configuration parameters) throws Exception {
                    ValueStateDescriptor<JSONObject> valueStateDescriptor
                        = new ValueStateDescriptor<JSONObject>("lastValueState", JSONObject.class);
                    lastValueState = getRuntimeContext().getState(valueStateDescriptor);
                }

                @Override
                public void processElement(JSONObject jsonObj, Context ctx, Collector<JSONObject> out) throws Exception {
                    //从状态中获取数据
                    JSONObject lastValue = lastValueState.value();
                    if (lastValue != null) {
                        //说明状态中已经有值了 ，出现了重复
                        //注意：这里写的是伪代码，时间应该是聚合的时间
                        Long ts1 = jsonObj.getLong("ts");
                        Long ts2 = lastValue.getLong("ts");
                        if (ts1 > ts2) {
                            lastValueState.update(jsonObj);
                        }
                    } else {
                        //说明状态中还没有值了 ，还没有重复
                        //将当前数据放到状态中
                        lastValueState.update(jsonObj);
                        //同时注册一个5s后执行的定时器
                        long currentProcessingTime = ctx.timerService().currentProcessingTime();
                        ctx.timerService().registerProcessingTimeTimer(currentProcessingTime + 5000L);
                    }
                }

                @Override
                public void onTimer(long timestamp, OnTimerContext ctx, Collector<JSONObject> out) throws Exception {
                    //定时器被触发的时候，执行的方法   将状态中的数据传递到下游
                    JSONObject jsonObj = lastValueState.value();
                    out.collect(jsonObj);
                    lastValueState.clear();
                }
            }
        );*/
        //5.3 状态 + 抵消
        SingleOutputStreamOperator<JSONObject> distinctDS = orderDetailIdKeyedDS.process(
            new KeyedProcessFunction<String, JSONObject, JSONObject>() {
                private ValueState<JSONObject> lastValueState;

                @Override
                public void open(Configuration parameters) throws Exception {
                    ValueStateDescriptor<JSONObject> valueStateDescriptor
                        = new ValueStateDescriptor<JSONObject>("lastValueState", JSONObject.class);
                    valueStateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(10)).build());
                    lastValueState = getRuntimeContext().getState(valueStateDescriptor);
                }

                @Override
                public void processElement(JSONObject jsonObj, Context ctx, Collector<JSONObject> out) throws Exception {
                    JSONObject lastValue = lastValueState.value();
                    if (lastValue != null) {
                        //如果状态中的数据不为空，说明重复了，如果重复的话，将状态中存储的数据影响到度量的属性进行取反，
                        //传递到下游，达到抵消的效果
                        String splitOriginalAmount = lastValue.getString("split_original_amount");
                        String splitTotalAmount = lastValue.getString("split_total_amount");
                        String splitCouponAmount = lastValue.getString("split_coupon_amount");
                        String splitActivityAmount = lastValue.getString("split_activity_amount");
                        lastValue.put("split_original_amount", "-" + splitOriginalAmount);
                        lastValue.put("split_total_amount", "-" + splitTotalAmount);
                        lastValue.put("split_coupon_amount", "-" + splitCouponAmount);
                        lastValue.put("split_activity_amount", "-" + splitActivityAmount);
                        out.collect(lastValue);
                    }
                    out.collect(jsonObj);
                    lastValueState.update(jsonObj);
                }
            }
        );

        //TODO 6.将流中类型 jsonObj->实体类对象
        SingleOutputStreamOperator<TradeSkuOrderBean> orderBeanDS = distinctDS.map(
            new MapFunction<JSONObject, TradeSkuOrderBean>() {
                @Override
                public TradeSkuOrderBean map(JSONObject jsonObj) throws Exception {
                    // {"create_time":"2023-06-05 08:51:37","sku_num":"1","activity_rule_id":"5","split_original_amount":"11999.0000",
                    // "split_coupon_amount":"0.0","sku_id":"19","date_id":"2023-06-05","user_id":"2326","province_id":"34",
                    // "activity_id":"4","sku_name":"TCL 8","id":"14256542","order_id":"53950","split_activity_amount":"1199.9",
                    // "split_total_amount":"10799.1","ts":"1686963097"}
                    String skuId = jsonObj.getString("sku_id");
                    String splitOriginalAmount = jsonObj.getString("split_original_amount");
                    String splitTotalAmount = jsonObj.getString("split_total_amount");
                    String splitActivityAmount = jsonObj.getString("split_activity_amount");
                    String splitCouponAmount = jsonObj.getString("split_coupon_amount");
                    Long ts = jsonObj.getLong("ts") * 1000L;
                    TradeSkuOrderBean orderBean = TradeSkuOrderBean.builder()
                        .skuId(skuId)
                        .originalAmount(new BigDecimal(splitOriginalAmount))
                        .orderAmount(new BigDecimal(splitTotalAmount))
                        .activityAmount(new BigDecimal(splitActivityAmount))
                        .couponAmount(new BigDecimal(splitCouponAmount))
                        .ts(ts)
                        .build();
                    return orderBean;
                }
            }
        );
        // orderBeanDS.print(">>>>");
        //TODO 7.指定Watermark以及提取事件时间字段
        SingleOutputStreamOperator<TradeSkuOrderBean> withWatermarkDS = orderBeanDS.assignTimestampsAndWatermarks(
            WatermarkStrategy
                .<TradeSkuOrderBean>forMonotonousTimestamps()
                .withTimestampAssigner(
                    new SerializableTimestampAssigner<TradeSkuOrderBean>() {
                        @Override
                        public long extractTimestamp(TradeSkuOrderBean orderBean, long recordTimestamp) {
                            return orderBean.getTs();
                        }
                    }
                )
        );
        //TODO 8.按照维度sku进行分组
        KeyedStream<TradeSkuOrderBean, String> skuIdKeyedDS
            = withWatermarkDS.keyBy(TradeSkuOrderBean::getSkuId);

        //TODO 9.开窗
        WindowedStream<TradeSkuOrderBean, String, TimeWindow> windowDS
            = skuIdKeyedDS.window(TumblingEventTimeWindows.of(org.apache.flink.streaming.api.windowing.time.Time.seconds(10)));

        //TODO 10.聚合计算
        SingleOutputStreamOperator<TradeSkuOrderBean> reduceDS = windowDS.reduce(
            new ReduceFunction<TradeSkuOrderBean>() {
                @Override
                public TradeSkuOrderBean reduce(TradeSkuOrderBean value1, TradeSkuOrderBean value2) throws Exception {
                    value1.setOriginalAmount(value1.getOriginalAmount().add(value2.getOriginalAmount()));
                    value1.setActivityAmount(value1.getActivityAmount().add(value2.getActivityAmount()));
                    value1.setCouponAmount(value1.getCouponAmount().add(value2.getCouponAmount()));
                    value1.setOrderAmount(value1.getOrderAmount().add(value2.getOrderAmount()));
                    return value1;
                }
            },
            new WindowFunction<TradeSkuOrderBean, TradeSkuOrderBean, String, TimeWindow>() {
                @Override
                public void apply(String s, TimeWindow window, Iterable<TradeSkuOrderBean> input, Collector<TradeSkuOrderBean> out) throws Exception {
                    String stt = DateFormatUtil.toYmdHms(window.getStart());
                    String edt = DateFormatUtil.toYmdHms(window.getEnd());
                    for (TradeSkuOrderBean orderBean : input) {
                        orderBean.setStt(stt);
                        orderBean.setEdt(edt);
                        orderBean.setTs(System.currentTimeMillis());
                        out.collect(orderBean);
                    }
                }
            }
        );
        // reduceDS.print(">>>");
        //TODO 11.关联商品SKU维度
        /*SingleOutputStreamOperator<TradeSkuOrderBean> withSkuInfoDS = reduceDS.map(
            new MapFunction<TradeSkuOrderBean, TradeSkuOrderBean>() {
                @Override
                public TradeSkuOrderBean map(TradeSkuOrderBean orderBean) throws Exception {
                    //根据流中的对象获取要关联的维度主键
                    String skuId = orderBean.getSkuId();
                    //根据维度的主键获取维度对象
                    JSONObject dimJsonObj = DimUtil.getDimInfo("dim_sku_info", skuId);
                    //将维度对象相关的属性赋值给流中的对象
                    // ID,SPU_ID,PRICE,SKU_NAME,SKU_DESC,WEIGHT,TM_ID,CATEGORY3_ID,SKU_DEFAULT_IMG,IS_SALE,CREATE_TIME
                    orderBean.setSkuName(dimJsonObj.getString("SKU_NAME"));
                    orderBean.setSpuId(dimJsonObj.getString("SPU_ID"));
                    orderBean.setTrademarkId(dimJsonObj.getString("TM_ID"));
                    orderBean.setCategory3Id(dimJsonObj.getString("CATEGORY3_ID"));
                    return orderBean;
                }
            }
        );
        withSkuInfoDS.print(">>>");*/
        // 将异步I/O操作应用于DataStream作为DataStream的一次转换操作
        /*SingleOutputStreamOperator<TradeSkuOrderBean> withSkuInfoDS = AsyncDataStream.unorderedWait(
            reduceDS,
            // 实现分发请求的 AsyncFunction
            new AsyncFunction<TradeSkuOrderBean, TradeSkuOrderBean>() {
                @Override
                public void asyncInvoke(TradeSkuOrderBean orderBean, ResultFuture<TradeSkuOrderBean> resultFuture) throws Exception {
                    //从线程池中获取线程
                    ThreadPoolExecutor poolExecutor = ThreadPoolUtil.getInstance();
                    poolExecutor.submit(
                        new Runnable() {
                            @Override
                            public void run() {
                                //根据流中对象获取要关联的维度主键
                                String skuId = orderBean.getSkuId();
                                //根据维度的主键获取维度对象
                                JSONObject dimJsonObj = DimUtil.getDimInfo("dim_sku_info", skuId);
                                //将维度对象的属性 补充到流中对象上
                                if(dimJsonObj != null){
                                    orderBean.setSkuName(dimJsonObj.getString("SKU_NAME"));
                                    orderBean.setSpuId(dimJsonObj.getString("SPU_ID"));
                                    orderBean.setTrademarkId(dimJsonObj.getString("TM_ID"));
                                    orderBean.setCategory3Id(dimJsonObj.getString("CATEGORY3_ID"));
                                }
                                //获取数据库交互的结果并发送给 ResultFuture的回调函数(将关联维度后的数据传递到下游)
                                resultFuture.complete(Collections.singleton(orderBean));
                            }
                        }
                    );
                }
            },
            60,
            TimeUnit.SECONDS
        );
        // withSkuInfoDS.print(">>>");
        */
        SingleOutputStreamOperator<TradeSkuOrderBean> withSkuInfoDS = AsyncDataStream.unorderedWait(
            reduceDS,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_sku_info") {
                @Override
                public void join(TradeSkuOrderBean orderBean, JSONObject dimJsonObj) {
                    orderBean.setSkuName(dimJsonObj.getString("SKU_NAME"));
                    orderBean.setSpuId(dimJsonObj.getString("SPU_ID"));
                    orderBean.setTrademarkId(dimJsonObj.getString("TM_ID"));
                    orderBean.setCategory3Id(dimJsonObj.getString("CATEGORY3_ID"));
                }

                @Override
                public String getKey(TradeSkuOrderBean orderBean) {
                    return orderBean.getSkuId();
                }
            },
            60, TimeUnit.SECONDS
        );
        // withSkuInfoDS.print(">>>>");


        //TODO 12.关联商品SPU维度
        SingleOutputStreamOperator<TradeSkuOrderBean> withSpuInfoDS = AsyncDataStream.unorderedWait(
            withSkuInfoDS,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_spu_info") {
                @Override
                public void join(TradeSkuOrderBean orderBean, JSONObject dimJsonObj) {
                    orderBean.setSpuName(dimJsonObj.getString("SPU_NAME"));
                }

                @Override
                public String getKey(TradeSkuOrderBean orderBean) {
                    return orderBean.getSpuId();
                }
            },
            60, TimeUnit.SECONDS
        );
        // withSpuInfoDS.print(">>>>");

        //TODO 13.关联商品TM维度
        SingleOutputStreamOperator<TradeSkuOrderBean> withTmDS = AsyncDataStream.unorderedWait(
            withSpuInfoDS,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_base_trademark") {
                @Override
                public void join(TradeSkuOrderBean orderBean, JSONObject dimJsonObj) {
                    orderBean.setTrademarkName(dimJsonObj.getString("TM_NAME"));
                }

                @Override
                public String getKey(TradeSkuOrderBean orderBean) {
                    return orderBean.getTrademarkId();
                }
            },
            60, TimeUnit.SECONDS
        );
        //TODO 14.关联商品category3维度
        SingleOutputStreamOperator<TradeSkuOrderBean> withCategory3Stream = AsyncDataStream.unorderedWait(
            withTmDS,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_base_category3") {
                @Override
                public void join(TradeSkuOrderBean javaBean,JSONObject jsonObj)  {
                    javaBean.setCategory3Name(jsonObj.getString("name".toUpperCase()));
                    javaBean.setCategory2Id(jsonObj.getString("category2_id".toUpperCase()));
                }

                @Override
                public String getKey(TradeSkuOrderBean javaBean) {
                    return javaBean.getCategory3Id();
                }
            },
            5 * 60, TimeUnit.SECONDS
        );

        //TODO 15.关联商品category2维度
        SingleOutputStreamOperator<TradeSkuOrderBean> withCategory2Stream = AsyncDataStream.unorderedWait(
            withCategory3Stream,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_base_category2".toUpperCase()) {
                @Override
                public void join(TradeSkuOrderBean javaBean,JSONObject jsonObj)  {
                    javaBean.setCategory2Name(jsonObj.getString("name".toUpperCase()));
                    javaBean.setCategory1Id(jsonObj.getString("category1_id".toUpperCase()));
                }

                @Override
                public String getKey(TradeSkuOrderBean javaBean) {
                    return javaBean.getCategory2Id();
                }
            },
            5 * 60, TimeUnit.SECONDS
        );

        //TODO 16.关联商品category1维度
        SingleOutputStreamOperator<TradeSkuOrderBean> withCategory1Stream = AsyncDataStream.unorderedWait(
            withCategory2Stream,
            new DimAsyncFunction<TradeSkuOrderBean>("dim_base_category1".toUpperCase()) {
                @Override
                public void join(TradeSkuOrderBean javaBean,JSONObject jsonObj)  {
                    javaBean.setCategory1Name(jsonObj.getString("name".toUpperCase()));
                }

                @Override
                public String getKey(TradeSkuOrderBean javaBean) {
                    return javaBean.getCategory1Id();
                }
            },
            5 * 60, TimeUnit.SECONDS
        );

        //TODO 17.将关联结果写到CK
        withCategory1Stream.print(">>>>>");
        withCategory1Stream.addSink(
            MyClickhouseUtil.getJdbcSink("insert into dws_trade_sku_order_window values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
        );

        env.execute();
    }
}
